package com.thinkgem.jeesite.modules.spider.processor;

import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.thinkgem.jeesite.modules.cms.entity.Article;
import com.thinkgem.jeesite.modules.cms.entity.ArticleData;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

public class CnblogsProcessor implements PageProcessor {
	private Logger logger = LoggerFactory.getLogger(getClass());
	private Site site;

	@Override
	public Site getSite() {
		site = Site.me().setRetryTimes(3) // 就我的经验,这个重试一般用处不大,他是httpclient内部重试
				.setTimeOut(30000)// 在使用代理的情况下,这个需要设置,可以考虑调大线程数目
				.setSleepTime(0)// 使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
				.setCycleRetryTimes(3)// 这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
				.setUseGzip(true)
				// .setRetryTimes(3).setRetrySleepTime(3000).setSleepTime(1000).setTimeOut(8000)
				.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
				.addHeader("Accept-Encoding", "gzip, deflate, br")
				.addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
				.addHeader("Cache-Control", "no-cache").addHeader("Connection", "keep-alive")
				.addHeader("Pragma", "no-cache").addHeader("Upgrade-Insecure-Requests", "1").addHeader("User-Agent",
						"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0");
		return site;
	}

	@Override
	public void process(Page page) {
		Html html = page.getHtml();
		String url = page.getUrl().toString();

		List<String> detailInfoList = html.xpath("//div[@class='post_item_body']/h3/a/@href").all();
		if (detailInfoList.size() != 0) {
			List<String> pageList = html.xpath("//div[@class='pager']/a/@href").all();
			for (String pageUrl : pageList) {
				page.addTargetRequest(pageUrl);
			}
			for (String userUrl : detailInfoList) {
				page.addTargetRequest(userUrl);
			}
			page.setSkip(true);
		} else {
			String title = html.xpath("//a[@id='cb_post_title_url']/text()").toString();
			String content = html.xpath("//div[@id='cnblogs_post_body']").toString();

			Article article = new Article();
			article.setTitle(title);
			ArticleData articleData = new ArticleData();
			articleData.setContent(content);
			articleData.setCopyfrom(url);
			articleData.setAllowComment("1");
			page.putField("article", article);
			page.putField("articleData", articleData);
		}
	}

	public static void main(String[] args) {
		Spider.create(new CnblogsProcessor())
				// .setDownloader(new DungProxyDownloader())
				//.thread(10).addUrl("http://www.cnblogs.com/cate/java/").run();
		 .addUrl("http://www.cnblogs.com/jing-an-feng-shao/p/5941098.html").run();
	}
}
